{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# from utils import *\n",
    "import tensorflow as tf\n",
    "import os\n",
    "import sklearn.datasets\n",
    "import numpy as np\n",
    "import re\n",
    "import collections\n",
    "import random\n",
    "from sklearn import metrics\n",
    "import jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 写入停用词\n",
    "with open(r'stopwords.txt','r',encoding='utf-8') as f:\n",
    "    english_stopwords = f.read().split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def separate_dataset(trainset, ratio = 0.5):\n",
    "    datastring = []\n",
    "    datatarget = []\n",
    "    for i in range(len(trainset.data)):\n",
    "        # 提取每一条文本数据，并过滤None值行文本；\n",
    "        data_ = trainset.data[i].split('\\n')\n",
    "        data_ = list(filter(None, data_))\n",
    "        # 抽取len(data_) * ratio个样本，并打乱某类样本顺序；\n",
    "        data_ = random.sample(data_, int(len(data_) * ratio))\n",
    "        # 去除停用词\n",
    "        for n in range(len(data_)):\n",
    "            data_[n] = clearstring(data_[n])\n",
    "        # 提取所有的词\n",
    "        datastring += data_\n",
    "        # 为每一个样本补上标签\n",
    "        for n in range(len(data_)):\n",
    "            datatarget.append(trainset.target[i])\n",
    "    return datastring, datatarget\n",
    "\n",
    "def clearstring(string):\n",
    "    # 清洗样本，并去停用词\n",
    "    # 去除非中文字符\n",
    "    string = re.sub(r'^[\\u4e00-\\u9fa5a-zA-Z0-9]', '', string)\n",
    "    string = list(jieba.cut(string, cut_all=False))\n",
    "    string = filter(None, string)\n",
    "    string = [y.strip() for y in string if y.strip() not in english_stopwords]\n",
    "    string = ' '.join(string)\n",
    "    return string.lower()\n",
    "\n",
    "\n",
    "def str_idx(corpus, dic, maxlen, UNK = 3):\n",
    "    # 词典索引\n",
    "    X = np.zeros((len(corpus), maxlen))\n",
    "    for i in range(len(corpus)):\n",
    "        for no, k in enumerate(corpus[i].split()[:maxlen][::-1]):\n",
    "            X[i, -1 - no] = dic.get(k, UNK)\n",
    "    return X\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['.ipynb_checkpoints', 'Chinese']\n",
      "14342\n",
      "14342\n"
     ]
    }
   ],
   "source": [
    "trainset = sklearn.datasets.load_files(container_path = 'dataset', encoding = 'UTF-8')\n",
    "trainset.data, trainset.target = separate_dataset(trainset,1.0)\n",
    "print(trainset.target_names)\n",
    "print(len(trainset.data))\n",
    "print(len(trainset.target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "\n",
    "def build_dataset(words, n_words, atleast=1):\n",
    "    # 四种填充词\n",
    "    count = [['PAD', 0], ['GO', 1], ['EOS', 2], ['UNK', 3]]\n",
    "    counter = collections.Counter(words).most_common(n_words)\n",
    "    # 过滤那些只有一个字的字符\n",
    "    counter = [i for i in counter if i[1] >= atleast]\n",
    "    count.extend(counter)\n",
    "    dictionary = dict()\n",
    "    # 构建词的索引\n",
    "    for word, _ in count:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    for word in words:\n",
    "        # 如果字典中没有出现的词，用unk表示\n",
    "        index = dictionary.get(word, 3)\n",
    "        data.append(index)\n",
    "    # 翻转字典\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, dictionary, reversed_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "split = (' '.join(trainset.data)).split()\n",
    "# 去重后的所有单词集合，组成词典\n",
    "vocabulary_size = len(list(set(split)))\n",
    "# data为所有词的词索引，词典，反向词典\n",
    "data, dictionary, rev_dictionary = build_dataset(split, vocabulary_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45533"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dictionary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def build_char_dataset(words):\n",
    "    # 四种填充词\n",
    "    count = []\n",
    "    dictionary = dict()\n",
    "    # 构建词的索引\n",
    "    for word in words:\n",
    "        dictionary[word] = len(dictionary)\n",
    "    data = list()\n",
    "    for word in words:\n",
    "        # 如果字典中没有出现的词，用unk表示\n",
    "        index = dictionary.get(word, 3)\n",
    "        data.append(index)\n",
    "    # 翻转字典\n",
    "    reversed_dictionary = dict(zip(dictionary.values(), dictionary.keys()))\n",
    "    return data, dictionary, reversed_dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 构建所有中文汉字的字典,3912个汉字\n",
    "char_split = list(set(list(''.join(trainset.data))))\n",
    "length = len(char_split)\n",
    "char_data, char_dictionary, char_rev_dictionary = build_char_dataset(char_split)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 给中文字符编码\n",
    "class Vocabulary:\n",
    "    def __init__(self, dictionary, rev_dictionary):\n",
    "        self._dictionary = dictionary\n",
    "        self._rev_dictionary = rev_dictionary\n",
    "    \n",
    "    # 起始符\n",
    "    @property\n",
    "    def start_string(self):\n",
    "        return self._dictionary['GO']\n",
    "    \n",
    "    # 结束符\n",
    "    @property\n",
    "    def end_string(self):\n",
    "        return self._dictionary['EOS']\n",
    "    \n",
    "    # 未知单词\n",
    "    @property\n",
    "    def unk(self):\n",
    "        return self._dictionary['UNK']\n",
    "\n",
    "    @property\n",
    "    def size(self):\n",
    "        return len(self._dictionary)\n",
    "    \n",
    "    # 查询词语的数值索引\n",
    "    def word_to_id(self, word):\n",
    "        return self._dictionary.get(word, self.unk)\n",
    "    \n",
    "    # 通过索引反查词语\n",
    "    def id_to_word(self, cur_id):\n",
    "        return self._rev_dictionary.get(cur_id, self._rev_dictionary[3])\n",
    "    \n",
    "    # 将数字索引解码成字符串并拼接起来\n",
    "    def decode(self, cur_ids):\n",
    "        return ' '.join([self.id_to_word(cur_id) for cur_id in cur_ids])\n",
    "    \n",
    "    # 将字符串编码成数字索引\n",
    "    def encode(self, sentence, reverse = False, split = True):\n",
    "\n",
    "        if split:\n",
    "            sentence = sentence.split()\n",
    "        # 将文本转化为数字索引\n",
    "        word_ids = [self.word_to_id(cur_word) for cur_word in sentence]\n",
    "        \n",
    "        # 为所有的文本加上起始符和结束符，双向编码都支持\n",
    "        if reverse:\n",
    "            return np.array(\n",
    "                [self.end_string] + word_ids + [self.start_string],\n",
    "                dtype = np.int32,\n",
    "            )\n",
    "        else:\n",
    "            return np.array(\n",
    "                [self.start_string] + word_ids + [self.end_string],\n",
    "                dtype = np.int32,\n",
    "            )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 英文，数字，字符用自制转码，中文字符，从0开始进行编码,考虑到中文短语的长度一般不超过8，故可取max_length=10\n",
    "class UnicodeCharsVocabulary(Vocabulary):\n",
    "    def __init__(self, dictionary, rev_dictionary,char_dictionary, char_rev_dictionary, max_word_length, **kwargs):\n",
    "        super(UnicodeCharsVocabulary, self).__init__(\n",
    "            dictionary, rev_dictionary, **kwargs\n",
    "        )\n",
    "        # 最大单词长度\n",
    "        self._max_word_length = max_word_length\n",
    "        self._char_dictionary = char_dictionary\n",
    "        self._char_rev_dictionary = char_rev_dictionary\n",
    "        self.bos_char = 3912\n",
    "        self.eos_char = 3913\n",
    "        self.bow_char = 3914\n",
    "        self.eow_char = 3915\n",
    "        self.pad_char = 3916\n",
    "        self.unk_char = 3917\n",
    "        # 单词的数量\n",
    "        num_words = self.size\n",
    "        \n",
    "        # 构建字符级别的词典表，[num_words,max_word_length]\n",
    "        self._word_char_ids = np.zeros(\n",
    "            [num_words, max_word_length], dtype = np.int32\n",
    "        )\n",
    "        \n",
    "        # 构建bos和eos的mask，初始化一个_max_word_length的张量，全部用3916填充，第一个字符位用3914，第三个字符位用3915，\n",
    "        # 第二个字符作为输入进行传入\n",
    "        def _make_bos_eos(c):\n",
    "            r = np.zeros([self._max_word_length], dtype = np.int32)\n",
    "            r[:] = self.pad_char\n",
    "            r[0] = self.bow_char\n",
    "            r[1] = c\n",
    "            r[2] = self.eow_char\n",
    "            return r\n",
    "        \n",
    "        # 张量化\n",
    "        self.bos_chars = _make_bos_eos(self.bos_char)\n",
    "        self.eos_chars = _make_bos_eos(self.eos_char)\n",
    "        \n",
    "        # 遍历字典中的每个单词，并将每个单词都进行字符级别的编码\n",
    "        for i, word in enumerate(self._dictionary.keys()):\n",
    "            self._word_char_ids[i] = self._convert_word_to_char_ids(word)\n",
    "        # 对于起始符GO和结束符EOS进行编码\n",
    "        self._word_char_ids[self.start_string] = self.bos_chars\n",
    "        self._word_char_ids[self.end_string] = self.eos_chars\n",
    "\n",
    "    @property\n",
    "    def word_char_ids(self):\n",
    "        return self._word_char_ids\n",
    "\n",
    "    @property\n",
    "    def max_word_length(self):\n",
    "        return self._max_word_length\n",
    "    \n",
    "    # 将单词转化为字符级别的索引\n",
    "    def _convert_word_to_char_ids(self, word):\n",
    "        # 对输入的单词进行张量化\n",
    "        code = np.zeros([self.max_word_length], dtype = np.int32)\n",
    "        code[:] = self.pad_char\n",
    "        # 截取maxlen-2个字符,并将所有字符转化为unicode字符集\n",
    "\n",
    "        word_encoded = [self._char_dictionary.get(item,self.unk_char) for item in list(word)][:(self.max_word_length - 2)]\n",
    "        # 第一个字符位为3914\n",
    "        code[0] = self.bow_char\n",
    "        # 遍历单词的每一个字符,k从1开始\n",
    "        for k, chr_id in enumerate(word_encoded, start = 1):\n",
    "            code[k] = chr_id\n",
    "        # 在单词的末尾补充一个单词末尾结束符3915\n",
    "        code[len(word_encoded) + 1] = self.eow_char\n",
    "        return code\n",
    "    \n",
    "    # 将单词转化为自定义字符编码\n",
    "    def word_to_char_ids(self, word):\n",
    "        if word in self._dictionary:\n",
    "            return self._word_char_ids[self._dictionary[word]]\n",
    "        else:\n",
    "            return self._convert_word_to_char_ids(word)\n",
    "    # 将句子转化为自定义字符编码矩阵\n",
    "    def encode_chars(self, sentence, reverse = False, split = True):\n",
    "        if split:\n",
    "            sentence = sentence.split()\n",
    "        chars_ids = [self.word_to_char_ids(cur_word) for cur_word in sentence]\n",
    "\n",
    "        if reverse:\n",
    "            return np.vstack([self.eos_chars] + chars_ids + [self.bos_chars])\n",
    "        else:\n",
    "            return np.vstack([self.bos_chars] + chars_ids + [self.eos_chars])\n",
    "\n",
    "\n",
    "def _get_batch(generator, batch_size, num_steps, max_word_length):\n",
    "    # generator: 生成器\n",
    "    # batch_size: 每个批次的字符串的数量\n",
    "    # num_steps: 窗口大小\n",
    "    # max_word_length: 最大单词长度，一般设置为50\n",
    "    # 初始化batch_size个字符串\n",
    "    cur_stream = [None] * batch_size\n",
    "\n",
    "    no_more_data = False\n",
    "    while True:\n",
    "        # 初始化单词矩阵输入0值化[batch_size,num_steps]\n",
    "        inputs = np.zeros([batch_size, num_steps], np.int32)\n",
    "        # 初始化字符级矩阵，输入0值化\n",
    "        if max_word_length is not None:\n",
    "            char_inputs = np.zeros(\n",
    "                [batch_size, num_steps, max_word_length], np.int32\n",
    "            )\n",
    "        else:\n",
    "            char_inputs = None\n",
    "        # 初始化单词矩阵输出0值化[batch_size,num_steps]\n",
    "        targets = np.zeros([batch_size, num_steps], np.int32)\n",
    "        for i in range(batch_size):\n",
    "            cur_pos = 0\n",
    "            while cur_pos < num_steps:\n",
    "                if cur_stream[i] is None or len(cur_stream[i][0]) <= 1:\n",
    "                    try:\n",
    "                        # 每一步都获取词索引，字符集编码器\n",
    "                        cur_stream[i] = list(next(generator))\n",
    "                    except StopIteration:\n",
    "                        no_more_data = True\n",
    "                        break\n",
    "                # how_many 取当前总num_steps与文本词向量数量的较小值，累加\n",
    "                how_many = min(len(cur_stream[i][0]) - 1, num_steps - cur_pos)\n",
    "                next_pos = cur_pos + how_many\n",
    "                \n",
    "                # 赋值输入对应的词索引范围和字符级别索引范围\n",
    "                inputs[i, cur_pos:next_pos] = cur_stream[i][0][:how_many]\n",
    "                if max_word_length is not None:\n",
    "                    char_inputs[i, cur_pos:next_pos] = cur_stream[i][1][\n",
    "                        :how_many\n",
    "                    ]\n",
    "                # targets 我们的目标是预测下一个词来优化emlo，所以我们以向右滑动的1个词作为target，作为预测对象\n",
    "                targets[i, cur_pos:next_pos] = cur_stream[i][0][\n",
    "                    1 : how_many + 1\n",
    "                ]\n",
    "\n",
    "                cur_pos = next_pos\n",
    "                \n",
    "                # 处理完之前那段，重新处理下一段，每段的长度取决于howmany,这里既是window的宽度。\n",
    "                cur_stream[i][0] = cur_stream[i][0][how_many:]\n",
    "                if max_word_length is not None:\n",
    "                    cur_stream[i][1] = cur_stream[i][1][how_many:]\n",
    "\n",
    "        if no_more_data:\n",
    "            break\n",
    "\n",
    "        X = {\n",
    "            'token_ids': inputs,\n",
    "            'tokens_characters': char_inputs,\n",
    "            'next_token_id': targets,\n",
    "        }\n",
    "\n",
    "        yield X\n",
    "\n",
    "\n",
    "class LMDataset:\n",
    "    def __init__(self, string, vocab, reverse = False):\n",
    "        self._vocab = vocab\n",
    "        self._string = string\n",
    "        self._reverse = reverse\n",
    "        self._use_char_inputs = hasattr(vocab, 'encode_chars')\n",
    "        self._i = 0\n",
    "        # 总文本的数量\n",
    "        self._nids = len(self._string)\n",
    "\n",
    "    def _load_string(self, string):\n",
    "        if self._reverse:\n",
    "            string = string.split()\n",
    "            string.reverse()\n",
    "            string = ' '.join(string)\n",
    "        \n",
    "        # 将一段文本解析成词索引，会在起始和末尾增加一个标志位\n",
    "        ids = self._vocab.encode(string, self._reverse)\n",
    "        \n",
    "        # 将一段文本解析成字符级编码\n",
    "        if self._use_char_inputs:\n",
    "            chars_ids = self._vocab.encode_chars(string, self._reverse)\n",
    "        else:\n",
    "            chars_ids = None\n",
    "        # 返回由词索引和字符集编码的元组\n",
    "        return list(zip([ids], [chars_ids]))[0]\n",
    "    \n",
    "    # 生成器，循环生成每个样本的词索引和字符编码\n",
    "    def get_sentence(self):\n",
    "        while True:\n",
    "            if self._i == self._nids:\n",
    "                self._i = 0\n",
    "            ret = self._load_string(self._string[self._i])\n",
    "            self._i += 1\n",
    "            yield ret\n",
    "\n",
    "    @property\n",
    "    def max_word_length(self):\n",
    "        if self._use_char_inputs:\n",
    "            return self._vocab.max_word_length\n",
    "        else:\n",
    "            return None\n",
    "    # batch生成器，每次只拿batch_size个数据，要多少数据就即时处理多少数据\n",
    "    def iter_batches(self, batch_size, num_steps):\n",
    "        for X in _get_batch(\n",
    "            self.get_sentence(), batch_size, num_steps, self.max_word_length\n",
    "        ):\n",
    "            yield X\n",
    "\n",
    "    @property\n",
    "    def vocab(self):\n",
    "        return self._vocab\n",
    "\n",
    "# 双向编码\n",
    "class BidirectionalLMDataset:\n",
    "    def __init__(self, string, vocab):\n",
    "        # 正向编码和反向编码\n",
    "        self._data_forward = LMDataset(string, vocab, reverse = False)\n",
    "        self._data_reverse = LMDataset(string, vocab, reverse = True)\n",
    "\n",
    "    def iter_batches(self, batch_size, num_steps):\n",
    "        max_word_length = self._data_forward.max_word_length\n",
    "\n",
    "        for X, Xr in zip(\n",
    "            _get_batch(\n",
    "                self._data_forward.get_sentence(),\n",
    "                batch_size,\n",
    "                num_steps,\n",
    "                max_word_length,\n",
    "            ),\n",
    "            _get_batch(\n",
    "                self._data_reverse.get_sentence(),\n",
    "                batch_size,\n",
    "                num_steps,\n",
    "                max_word_length,\n",
    "            ),\n",
    "        ):\n",
    "            # 拼接成一个6个item的字典，前三个为正向，后三个为反向\n",
    "            for k, v in Xr.items():\n",
    "                X[k + '_reverse'] = v\n",
    "\n",
    "            yield X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# maxlens=10，很明显没有超过8的词语，其中两个用来做填充符\n",
    "uni = UnicodeCharsVocabulary(dictionary, rev_dictionary,char_dictionary,char_rev_dictionary, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bi = BidirectionalLMDataset(trainset.data, uni)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 每次只输入16个样本数据\n",
    "batch_size = 16\n",
    "# 训练用的词典大小\n",
    "n_train_tokens = len(dictionary)\n",
    "# 语言模型参数配置项\n",
    "options = {\n",
    "    # 开启双向编码机制\n",
    "    'bidirectional': True,\n",
    "    # 字符级别的CNN，字符级别词嵌入128维，一共配置7种类型的滤波器，每个词最大长度为50，编码的有效数量为3918个，设置两条高速通道\n",
    "    'char_cnn': {\n",
    "        'activation': 'relu',\n",
    "        'embedding': {'dim': 128},\n",
    "        'filters': [\n",
    "            [1, 32],\n",
    "            [2, 32],\n",
    "            [3, 64],\n",
    "            [4, 128],\n",
    "            [5, 256],\n",
    "            [6, 512],\n",
    "            [7, 1024],\n",
    "        ],\n",
    "        'max_characters_per_token': 10,\n",
    "        'n_characters': 3918,\n",
    "        'n_highway': 2,\n",
    "    },\n",
    "    # 随机失活率设置为0.1\n",
    "    'dropout': 0.1,\n",
    "    # lstm单元，设置三层，嵌入维度为512维\n",
    "    'lstm': {\n",
    "        # 截断值\n",
    "        'cell_clip': 3,\n",
    "        'dim': 512,\n",
    "        'n_layers': 2,\n",
    "        'projection_dim': 256,\n",
    "        # 裁剪到[-3,3]之间\n",
    "        'proj_clip': 3,\n",
    "        'use_skip_connections': True,\n",
    "    },\n",
    "    # 一共迭代100轮\n",
    "    'n_epochs': 100,\n",
    "    # 训练词典的大小\n",
    "    'n_train_tokens': n_train_tokens,\n",
    "    # 每个batch的大小\n",
    "    'batch_size': batch_size,\n",
    "    # 所有词的数量\n",
    "    'n_tokens_vocab': uni.size,\n",
    "    # 推断区间为20\n",
    "    'unroll_steps': 20,\n",
    "    'n_negative_samples_batch': 0.001,\n",
    "    'sample_softmax': True,\n",
    "    'share_embedding_softmax': False,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# 构建ELMO语言模型\n",
    "class LanguageModel:\n",
    "    def __init__(self, options, is_training):\n",
    "        self.options = options\n",
    "        self.is_training = is_training\n",
    "        self.bidirectional = options.get('bidirectional', False)\n",
    "\n",
    "        self.char_inputs = 'char_cnn' in self.options\n",
    "\n",
    "        self.share_embedding_softmax = options.get(\n",
    "            'share_embedding_softmax', False\n",
    "        )\n",
    "        if self.char_inputs and self.share_embedding_softmax:\n",
    "            raise ValueError(\n",
    "                'Sharing softmax and embedding weights requires ' 'word input'\n",
    "            )\n",
    "\n",
    "        self.sample_softmax = options.get('sample_softmax', False)\n",
    "        # 建立模型\n",
    "        self._build()\n",
    "        # 配置学习率\n",
    "        lr = options.get('learning_rate', 0.2)\n",
    "        # 配置优化器\n",
    "        self.optimizer = tf.train.AdagradOptimizer(\n",
    "            learning_rate = lr, initial_accumulator_value = 1.0\n",
    "        ).minimize(self.total_loss)\n",
    "\n",
    "    def _build_word_embeddings(self):\n",
    "        # 建立词嵌入\n",
    "        # 加载所有的词\n",
    "        n_tokens_vocab = self.options['n_tokens_vocab']\n",
    "        batch_size = self.options['batch_size']\n",
    "        # 上下文推断的窗口大小，这里关联20个单词\n",
    "        unroll_steps = self.options['unroll_steps']\n",
    "        # 词嵌入维度128\n",
    "        projection_dim = self.options['lstm']['projection_dim']\n",
    "        # 词索引\n",
    "        self.token_ids = tf.placeholder(\n",
    "            tf.int32, shape = (None, unroll_steps), name = 'token_ids'\n",
    "        )\n",
    "        self.batch_size = tf.shape(self.token_ids)[0]\n",
    "        with tf.device('/cpu:0'):\n",
    "            \n",
    "            # 对单词进行256维的单词编码，初始化数据服从(-1,1)的正态分布\n",
    "            self.embedding_weights = tf.get_variable(\n",
    "                'embedding',\n",
    "                [n_tokens_vocab, projection_dim],\n",
    "                dtype = tf.float32,\n",
    "                initializer = tf.random_uniform_initializer(-1.0, 1.0),\n",
    "            )\n",
    "            # 20个词对应的词嵌入\n",
    "            self.embedding = tf.nn.embedding_lookup(\n",
    "                self.embedding_weights, self.token_ids\n",
    "            )\n",
    "        \n",
    "        # 启用双向编码机制\n",
    "        if self.bidirectional:\n",
    "            self.token_ids_reverse = tf.placeholder(\n",
    "                tf.int32,\n",
    "                shape = (None, unroll_steps),\n",
    "                name = 'token_ids_reverse',\n",
    "            )\n",
    "            with tf.device('/cpu:0'):\n",
    "                self.embedding_reverse = tf.nn.embedding_lookup(\n",
    "                    self.embedding_weights, self.token_ids_reverse\n",
    "                )\n",
    "\n",
    "    def _build_word_char_embeddings(self):\n",
    "\n",
    "        batch_size = self.options['batch_size']\n",
    "        unroll_steps = self.options['unroll_steps']\n",
    "        projection_dim = self.options['lstm']['projection_dim']\n",
    "\n",
    "        cnn_options = self.options['char_cnn']\n",
    "        filters = cnn_options['filters']\n",
    "        # 求和所有的滤波器数量\n",
    "        n_filters = sum(f[1] for f in filters)\n",
    "        # 最大单词字符长度\n",
    "        max_chars = cnn_options['max_characters_per_token']\n",
    "        # 字符级别嵌入维度，128\n",
    "        char_embed_dim = cnn_options['embedding']['dim']\n",
    "        # 所有字符的类型，一共261种\n",
    "        n_chars = cnn_options['n_characters']\n",
    "        \n",
    "        # 配置激活函数\n",
    "        if cnn_options['activation'] == 'tanh':\n",
    "            activation = tf.nn.tanh\n",
    "        elif cnn_options['activation'] == 'relu':\n",
    "            activation = tf.nn.relu\n",
    "        \n",
    "        # [batch_size,unroll_steps,max_chars]\n",
    "        self.tokens_characters = tf.placeholder(\n",
    "            tf.int32,\n",
    "            shape = (None, unroll_steps, max_chars),\n",
    "            name = 'tokens_characters',\n",
    "        )\n",
    "        self.batch_size = tf.shape(self.tokens_characters)[0]\n",
    "        with tf.device('/cpu:0'):\n",
    "            # 字符级别词嵌入，嵌入维度128维\n",
    "            self.embedding_weights = tf.get_variable(\n",
    "                'char_embed',\n",
    "                [n_chars, char_embed_dim],\n",
    "                dtype = tf.float32,\n",
    "                initializer = tf.random_uniform_initializer(-1.0, 1.0),\n",
    "            )\n",
    "            self.char_embedding = tf.nn.embedding_lookup(\n",
    "                self.embedding_weights, self.tokens_characters\n",
    "            )\n",
    "\n",
    "            if self.bidirectional:\n",
    "                self.tokens_characters_reverse = tf.placeholder(\n",
    "                    tf.int32,\n",
    "                    shape = (None, unroll_steps, max_chars),\n",
    "                    name = 'tokens_characters_reverse',\n",
    "                )\n",
    "                self.char_embedding_reverse = tf.nn.embedding_lookup(\n",
    "                    self.embedding_weights, self.tokens_characters_reverse\n",
    "                )\n",
    "                \n",
    "        # 构建卷积层网络，用于字符级别的CNN卷积\n",
    "        def make_convolutions(inp, reuse):\n",
    "            with tf.variable_scope('CNN', reuse = reuse) as scope:\n",
    "                convolutions = []\n",
    "                # 这里构建7层卷积网络\n",
    "                for i, (width, num) in enumerate(filters):\n",
    "                    if cnn_options['activation'] == 'relu':\n",
    "                        w_init = tf.random_uniform_initializer(\n",
    "                            minval = -0.05, maxval = 0.05\n",
    "                        )\n",
    "                    elif cnn_options['activation'] == 'tanh':\n",
    "                        w_init = tf.random_normal_initializer(\n",
    "                            mean = 0.0,\n",
    "                            stddev = np.sqrt(1.0 / (width * char_embed_dim)),\n",
    "                        )\n",
    "                    w = tf.get_variable(\n",
    "                        'W_cnn_%s' % i,\n",
    "                        [1, width, char_embed_dim, num],\n",
    "                        initializer = w_init,\n",
    "                        dtype = tf.float32,\n",
    "                    )\n",
    "                    b = tf.get_variable(\n",
    "                        'b_cnn_%s' % i,\n",
    "                        [num],\n",
    "                        dtype = tf.float32,\n",
    "                        initializer = tf.constant_initializer(0.0),\n",
    "                    )\n",
    "                    # 卷积，uroll_nums,characters_nums采用1*1，1*2，...，1*7的卷积核，采用valid卷积策略；\n",
    "                    # width上，(uroll_nums-1/1)+1=uroll_nums\n",
    "                    # height上，(characters_nums-7/1)+1，捕捉词与词之间的相关性\n",
    "                    conv = (\n",
    "                        tf.nn.conv2d(\n",
    "                            inp, w, strides = [1, 1, 1, 1], padding = 'VALID'\n",
    "                        )\n",
    "                        + b\n",
    "                    )\n",
    "                    # 最大池化，每个词的字符编码\n",
    "                    conv = tf.nn.max_pool(\n",
    "                        conv,\n",
    "                        [1, 1, max_chars - width + 1, 1],\n",
    "                        [1, 1, 1, 1],\n",
    "                        'VALID',\n",
    "                    )\n",
    "                    conv = activation(conv)\n",
    "                    # 删除第三维，输入为[batch_size,uroll_nums,1,nums]\n",
    "                    # 输出为[batch_size,uroll_nums,nums]\n",
    "                    conv = tf.squeeze(conv, squeeze_dims = [2])\n",
    "                    \n",
    "                    # 收集每个卷积层，并进行拼接\n",
    "                    convolutions.append(conv)\n",
    "\n",
    "            return tf.concat(convolutions, 2)\n",
    "\n",
    "        reuse = tf.get_variable_scope().reuse\n",
    "        # inp [batch_size,uroll_nums,characters_nums,embedding_size]\n",
    "        embedding = make_convolutions(self.char_embedding, reuse)\n",
    "        # [batch_size,20,2048] #经过验证无误\n",
    "        # 增加一维[1,batch_size,uroll_nums,nums++]\n",
    "        self.token_embedding_layers = [embedding]\n",
    "        if self.bidirectional:\n",
    "            embedding_reverse = make_convolutions(\n",
    "                self.char_embedding_reverse, True\n",
    "            )\n",
    "        # 高速网络的数量\n",
    "        n_highway = cnn_options.get('n_highway')\n",
    "        use_highway = n_highway is not None and n_highway > 0\n",
    "        # use_proj 为True\n",
    "        use_proj = n_filters != projection_dim\n",
    "        \n",
    "        # 本来已经第三维是2048维了，这么做的原因是？\n",
    "        if use_highway or use_proj:\n",
    "            embedding = tf.reshape(embedding, [-1, n_filters])\n",
    "            if self.bidirectional:\n",
    "                embedding_reverse = tf.reshape(\n",
    "                    embedding_reverse, [-1, n_filters]\n",
    "                )\n",
    "\n",
    "        if use_proj:\n",
    "            # 使用投影，将滤波器再投影到一个projection_dim维的向量空间内\n",
    "            assert n_filters > projection_dim\n",
    "            with tf.variable_scope('CNN_proj') as scope:\n",
    "                W_proj_cnn = tf.get_variable(\n",
    "                    'W_proj',\n",
    "                    [n_filters, projection_dim],\n",
    "                    initializer = tf.random_normal_initializer(\n",
    "                        mean = 0.0, stddev = np.sqrt(1.0 / n_filters)\n",
    "                    ),\n",
    "                    dtype = tf.float32,\n",
    "                )\n",
    "                b_proj_cnn = tf.get_variable(\n",
    "                    'b_proj',\n",
    "                    [projection_dim],\n",
    "                    initializer = tf.constant_initializer(0.0),\n",
    "                    dtype = tf.float32,\n",
    "                )\n",
    "\n",
    "        def high(x, ww_carry, bb_carry, ww_tr, bb_tr):\n",
    "            carry_gate = tf.nn.sigmoid(tf.matmul(x, ww_carry) + bb_carry)\n",
    "            transform_gate = tf.nn.relu(tf.matmul(x, ww_tr) + bb_tr)\n",
    "            return carry_gate * transform_gate + (1.0 - carry_gate) * x\n",
    "\n",
    "        if use_highway:\n",
    "            # 高速网络的维度为2048维\n",
    "            highway_dim = n_filters\n",
    "\n",
    "            for i in range(n_highway):\n",
    "                with tf.variable_scope('CNN_high_%s' % i) as scope:\n",
    "                    W_carry = tf.get_variable(\n",
    "                        'W_carry',\n",
    "                        [highway_dim, highway_dim],\n",
    "                        initializer = tf.random_normal_initializer(\n",
    "                            mean = 0.0, stddev = np.sqrt(1.0 / highway_dim)\n",
    "                        ),\n",
    "                        dtype = tf.float32,\n",
    "                    )\n",
    "                    b_carry = tf.get_variable(\n",
    "                        'b_carry',\n",
    "                        [highway_dim],\n",
    "                        initializer = tf.constant_initializer(-2.0),\n",
    "                        dtype = tf.float32,\n",
    "                    )\n",
    "                    W_transform = tf.get_variable(\n",
    "                        'W_transform',\n",
    "                        [highway_dim, highway_dim],\n",
    "                        initializer = tf.random_normal_initializer(\n",
    "                            mean = 0.0, stddev = np.sqrt(1.0 / highway_dim)\n",
    "                        ),\n",
    "                        dtype = tf.float32,\n",
    "                    )\n",
    "                    b_transform = tf.get_variable(\n",
    "                        'b_transform',\n",
    "                        [highway_dim],\n",
    "                        initializer = tf.constant_initializer(0.0),\n",
    "                        dtype = tf.float32,\n",
    "                    )\n",
    "\n",
    "                embedding = high(\n",
    "                    embedding, W_carry, b_carry, W_transform, b_transform\n",
    "                )\n",
    "                if self.bidirectional:\n",
    "                    embedding_reverse = high(\n",
    "                        embedding_reverse,\n",
    "                        W_carry,\n",
    "                        b_carry,\n",
    "                        W_transform,\n",
    "                        b_transform,\n",
    "                    )\n",
    "                # 扩展一层和两层经过高速网络的参数\n",
    "                self.token_embedding_layers.append(\n",
    "                    tf.reshape(\n",
    "                        embedding, [self.batch_size, unroll_steps, highway_dim]\n",
    "                    )\n",
    "                )\n",
    "        \n",
    "        # 经过一层线性变换[bacth_size,unroll_nums,projection_dim]\n",
    "        if use_proj:\n",
    "            embedding = tf.matmul(embedding, W_proj_cnn) + b_proj_cnn\n",
    "            if self.bidirectional:\n",
    "                embedding_reverse = (\n",
    "                    tf.matmul(embedding_reverse, W_proj_cnn) + b_proj_cnn\n",
    "                )\n",
    "            # 只经过线性变换的网络参数\n",
    "            self.token_embedding_layers.append(\n",
    "                tf.reshape(\n",
    "                    embedding, [self.batch_size, unroll_steps, projection_dim]\n",
    "                )\n",
    "            )\n",
    "        \n",
    "        # 确保矩阵尺寸相同\n",
    "        if use_highway or use_proj:\n",
    "            shp = [self.batch_size, unroll_steps, projection_dim]\n",
    "            embedding = tf.reshape(embedding, shp)\n",
    "            if self.bidirectional:\n",
    "                embedding_reverse = tf.reshape(embedding_reverse, shp)\n",
    "                \n",
    "        # 经过线性变化的embdedding [bacth_size,unroll_nums,projection_dim]\n",
    "        # self.token_embedding_layers 由四个嵌入层参数组成\n",
    "        # [bacth_size,unroll_nums,nums++] 原始词嵌入\n",
    "        # [bacth_size,unroll_nums,highway_dim] 经过第一层高速网络的词嵌入\n",
    "        # [bacth_size,unroll_nums,highway_dim] 经过第二层高速网络的词嵌入\n",
    "        # [bacth_size,unroll_nums,projection_dim] 经过低微线性投影的词嵌入\n",
    "        # print(embedding)\n",
    "        # print(self.token_embedding_layers)\n",
    "        self.embedding = embedding\n",
    "        if self.bidirectional:\n",
    "            self.embedding_reverse = embedding_reverse\n",
    "    \n",
    "    # 构建模型\n",
    "    def _build(self):\n",
    "        # 所有词的数量\n",
    "        n_tokens_vocab = self.options['n_tokens_vocab']\n",
    "        batch_size = self.options['batch_size']\n",
    "        # window长度\n",
    "        unroll_steps = self.options['unroll_steps']\n",
    "        \n",
    "        # lstm编码长度\n",
    "        lstm_dim = self.options['lstm']['dim']\n",
    "        projection_dim = self.options['lstm']['projection_dim']\n",
    "        # lstm的层数\n",
    "        n_lstm_layers = self.options['lstm'].get('n_layers', 1)\n",
    "        dropout = self.options['dropout']\n",
    "        # 保有率\n",
    "        keep_prob = 1.0 - dropout\n",
    "        \n",
    "        # 如果是字符级别的输入，则建立词，字符嵌入，否则建立词嵌入，实际上使用前者\n",
    "        if self.char_inputs:\n",
    "            self._build_word_char_embeddings()\n",
    "        else:\n",
    "            self._build_word_embeddings()\n",
    "        \n",
    "        # 存储lstm的状态\n",
    "        self.init_lstm_state = []\n",
    "        self.final_lstm_state = []\n",
    "        \n",
    "        # 双向\n",
    "        # lstm_inputs单元为[batch_size,uroll_nums,projection_dim]双向单元\n",
    "        if self.bidirectional:\n",
    "            lstm_inputs = [self.embedding, self.embedding_reverse]\n",
    "        else:\n",
    "            lstm_inputs = [self.embedding]\n",
    "\n",
    "        cell_clip = self.options['lstm'].get('cell_clip')\n",
    "        proj_clip = self.options['lstm'].get('proj_clip')\n",
    "\n",
    "        \n",
    "        use_skip_connections = self.options['lstm'].get('use_skip_connections')\n",
    "        print(lstm_inputs)\n",
    "        lstm_outputs = []\n",
    "        for lstm_num, lstm_input in enumerate(lstm_inputs):\n",
    "            lstm_cells = []\n",
    "            for i in range(n_lstm_layers):\n",
    "                # 在进行LSTM编码后再接入一个num_proj的全连接层，[batch_size,projection_dim]\n",
    "                # [batch_size,num_proj]\n",
    "                lstm_cell = tf.nn.rnn_cell.LSTMCell(\n",
    "                    # 隐含层的单元数\n",
    "                    lstm_dim,\n",
    "                    num_proj = lstm_dim // 2,\n",
    "                    cell_clip = cell_clip,\n",
    "                    proj_clip = proj_clip,\n",
    "                )\n",
    "\n",
    "                if use_skip_connections:\n",
    "                    if i == 0:\n",
    "                        pass\n",
    "                    else:\n",
    "                        # 将上一个单元的输出，和当前输入映射到下一个单元\n",
    "                        lstm_cell = tf.nn.rnn_cell.ResidualWrapper(lstm_cell)\n",
    "                \n",
    "                # 添加随机失活层\n",
    "                if self.is_training:\n",
    "                    lstm_cell = tf.nn.rnn_cell.DropoutWrapper(\n",
    "                        lstm_cell, input_keep_prob = keep_prob\n",
    "                    )\n",
    "\n",
    "                lstm_cells.append(lstm_cell)\n",
    "            \n",
    "            # 构建多层LSTM\n",
    "            if n_lstm_layers > 1:\n",
    "                lstm_cell = tf.nn.rnn_cell.MultiRNNCell(lstm_cells)\n",
    "            else:\n",
    "                lstm_cell = lstm_cells[0]\n",
    "\n",
    "            with tf.control_dependencies([lstm_input]):\n",
    "                # 初始化状态\n",
    "                self.init_lstm_state.append(\n",
    "                    lstm_cell.zero_state(self.batch_size, tf.float32)\n",
    "                )\n",
    "                if self.bidirectional:\n",
    "                    with tf.variable_scope('RNN_%s' % lstm_num):\n",
    "                        # 从最后一步开始，获取最后一步的输出，和最终的隐含状态,确保正反向LSTM单元可以拼接起来\n",
    "                        _lstm_output_unpacked, final_state = tf.nn.static_rnn(\n",
    "                            lstm_cell,\n",
    "                            # 将每个词对应的张量进行分离并作为LSTM的输入\n",
    "                            tf.unstack(lstm_input, axis = 1),\n",
    "                            initial_state = self.init_lstm_state[-1],\n",
    "                        )\n",
    "                else:\n",
    "                    _lstm_output_unpacked, final_state = tf.nn.static_rnn(\n",
    "                        lstm_cell,\n",
    "                        tf.unstack(lstm_input, axis = 1),\n",
    "                        initial_state = self.init_lstm_state[-1],\n",
    "                    )\n",
    "                self.final_lstm_state.append(final_state)\n",
    "            # [batch_size,num_proj]\n",
    "#             print(final_state)\n",
    "            # 将一个隐含层的输出拼接起来 [batch_size,20,256]\n",
    "            lstm_output_flat = tf.reshape(\n",
    "                tf.stack(_lstm_output_unpacked, axis = 1), [-1, projection_dim]\n",
    "            )\n",
    "            print(lstm_output_flat)\n",
    "            tf.add_to_collection(\n",
    "                'lstm_output_embeddings', _lstm_output_unpacked\n",
    "            )\n",
    "\n",
    "            lstm_outputs.append(lstm_output_flat)\n",
    "        self._build_loss(lstm_outputs)\n",
    "    \n",
    "    # 构建损失函数\n",
    "    def _build_loss(self, lstm_outputs):\n",
    "        batch_size = self.options['batch_size']\n",
    "        unroll_steps = self.options['unroll_steps']\n",
    "        \n",
    "        # 所有词的数量\n",
    "        n_tokens_vocab = self.options['n_tokens_vocab']\n",
    "\n",
    "        def _get_next_token_placeholders(suffix):\n",
    "            name = 'next_token_id' + suffix\n",
    "            id_placeholder = tf.placeholder(\n",
    "                tf.int32, shape = (None, unroll_steps), name = name\n",
    "            )\n",
    "            return id_placeholder\n",
    "\n",
    "        self.next_token_id = _get_next_token_placeholders('')\n",
    "        # 每次抽取[batch_size,unroll_nums]个词\n",
    "        print(self.next_token_id)\n",
    "        if self.bidirectional:\n",
    "            self.next_token_id_reverse = _get_next_token_placeholders(\n",
    "                '_reverse'\n",
    "            )\n",
    "        # softmax的维度为projection_dim（256）\n",
    "        softmax_dim = self.options['lstm']['projection_dim']\n",
    "        # 与词嵌入的权重共享\n",
    "        if self.share_embedding_softmax:\n",
    "            self.softmax_W = self.embedding_weights\n",
    "\n",
    "        # 初始化softmax的参数\n",
    "        with tf.variable_scope('softmax'), tf.device('/cpu:0'):\n",
    "            softmax_init = tf.random_normal_initializer(\n",
    "                0.0, 1.0 / np.sqrt(softmax_dim)\n",
    "            )\n",
    "            # softmax分布到每一个词中\n",
    "            if not self.share_embedding_softmax:\n",
    "                self.softmax_W = tf.get_variable(\n",
    "                    'W',\n",
    "                    [n_tokens_vocab, softmax_dim],\n",
    "                    dtype = tf.float32,\n",
    "                    initializer = softmax_init,\n",
    "                )\n",
    "            self.softmax_b = tf.get_variable(\n",
    "                'b',\n",
    "                [n_tokens_vocab],\n",
    "                dtype = tf.float32,\n",
    "                initializer = tf.constant_initializer(0.0),\n",
    "            )\n",
    "\n",
    "        self.individual_losses = []\n",
    "\n",
    "        if self.bidirectional:\n",
    "            next_ids = [self.next_token_id, self.next_token_id_reverse]\n",
    "        else:\n",
    "            next_ids = [self.next_token_id]\n",
    "        \n",
    "        print(lstm_outputs)\n",
    "        self.output_scores = tf.identity(lstm_outputs, name = 'softmax_score')\n",
    "        print(self.output_scores)\n",
    "        \n",
    "        for id_placeholder, lstm_output_flat in zip(next_ids, lstm_outputs):\n",
    "            next_token_id_flat = tf.reshape(id_placeholder, [-1, 1])\n",
    "            with tf.control_dependencies([lstm_output_flat]):\n",
    "                if self.is_training and self.sample_softmax:\n",
    "                    losses = tf.nn.sampled_softmax_loss(\n",
    "                        self.softmax_W,\n",
    "                        self.softmax_b,\n",
    "                        next_token_id_flat,\n",
    "                        lstm_output_flat,\n",
    "                        int(\n",
    "                            self.options['n_negative_samples_batch']\n",
    "                            * self.options['n_tokens_vocab']\n",
    "                        ),\n",
    "                        self.options['n_tokens_vocab'],\n",
    "                        num_true = 1,\n",
    "                    )\n",
    "\n",
    "                else:\n",
    "                    output_scores = (\n",
    "                        tf.matmul(\n",
    "                            lstm_output_flat, tf.transpose(self.softmax_W)\n",
    "                        )\n",
    "                        + self.softmax_b\n",
    "                    )\n",
    "\n",
    "                    losses = tf.nn.sparse_softmax_cross_entropy_with_logits(\n",
    "                        logits = self.output_scores,\n",
    "                        labels = tf.squeeze(\n",
    "                            next_token_id_flat, squeeze_dims = [1]\n",
    "                        ),\n",
    "                    )\n",
    "\n",
    "            self.individual_losses.append(tf.reduce_mean(losses))\n",
    "\n",
    "        if self.bidirectional:\n",
    "            self.total_loss = 0.5 * (\n",
    "                self.individual_losses[0] + self.individual_losses[1]\n",
    "            )\n",
    "        else:\n",
    "            self.total_loss = self.individual_losses[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0725 20:41:37.074845 42196 deprecation.py:506] From E:\\Anaconda\\envs\\git_test\\lib\\site-packages\\tensorflow\\python\\util\\dispatch.py:180: calling squeeze (from tensorflow.python.ops.array_ops) with squeeze_dims is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use the `axis` argument instead\n",
      "W0725 20:41:37.241371 42196 deprecation.py:323] From <ipython-input-29-70f0070b2d45>:358: LSTMCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.LSTMCell, and will be replaced by that in Tensorflow 2.0.\n",
      "W0725 20:41:37.247355 42196 deprecation.py:323] From <ipython-input-29-70f0070b2d45>:378: MultiRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n",
      "W0725 20:41:37.269296 42196 deprecation.py:323] From <ipython-input-29-70f0070b2d45>:394: static_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell, unroll=True)`, which is equivalent to this API\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<tf.Tensor 'Reshape_5:0' shape=(?, 20, 256) dtype=float32>, <tf.Tensor 'Reshape_6:0' shape=(?, 20, 256) dtype=float32>]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0725 20:41:37.531595 42196 deprecation.py:506] From E:\\Anaconda\\envs\\git_test\\lib\\site-packages\\tensorflow\\python\\ops\\init_ops.py:1251: calling VarianceScaling.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n",
      "W0725 20:41:37.539573 42196 deprecation.py:506] From E:\\Anaconda\\envs\\git_test\\lib\\site-packages\\tensorflow\\python\\ops\\rnn_cell_impl.py:961: calling Zeros.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"Reshape_7:0\", shape=(?, 256), dtype=float32)\n",
      "Tensor(\"Reshape_8:0\", shape=(?, 256), dtype=float32)\n",
      "Tensor(\"next_token_id:0\", shape=(?, 20), dtype=int32)\n",
      "[<tf.Tensor 'Reshape_7:0' shape=(?, 256) dtype=float32>, <tf.Tensor 'Reshape_8:0' shape=(?, 256) dtype=float32>]\n",
      "Tensor(\"softmax_score:0\", shape=(2, ?, 256), dtype=float32)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "W0725 20:41:40.985382 42196 deprecation.py:323] From E:\\Anaconda\\envs\\git_test\\lib\\site-packages\\tensorflow\\python\\ops\\math_grad.py:1250: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "W0725 20:41:48.755593 42196 deprecation.py:506] From E:\\Anaconda\\envs\\git_test\\lib\\site-packages\\tensorflow\\python\\training\\adagrad.py:76: calling Constant.__init__ (from tensorflow.python.ops.init_ops) with dtype is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Call initializer instance with the dtype argument instead of passing it to the constructor\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "model = LanguageModel(options, True)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "def _get_feed_dict_from_X(X, model, char_inputs, bidirectional):\n",
    "    feed_dict = {}\n",
    "    if not char_inputs:\n",
    "        token_ids = X['token_ids']\n",
    "        feed_dict[model.token_ids] = token_ids\n",
    "    else:\n",
    "        char_ids = X['tokens_characters']\n",
    "        feed_dict[model.tokens_characters] = char_ids\n",
    "    if bidirectional:\n",
    "        if not char_inputs:\n",
    "            feed_dict[model.token_ids_reverse] = X['token_ids_reverse']\n",
    "        else:\n",
    "            feed_dict[model.tokens_characters_reverse] = X['tokens_characters_reverse']\n",
    "    next_id_placeholders = [[model.next_token_id, '']]\n",
    "    if bidirectional:\n",
    "        next_id_placeholders.append([model.next_token_id_reverse, '_reverse'])\n",
    "\n",
    "    for id_placeholder, suffix in next_id_placeholders:\n",
    "        name = 'next_token_id' + suffix\n",
    "        feed_dict[id_placeholder] = X[name]\n",
    "\n",
    "    return feed_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "bidirectional = options.get('bidirectional', False)\n",
    "batch_size = options['batch_size']\n",
    "unroll_steps = options['unroll_steps']\n",
    "n_train_tokens = options.get('n_train_tokens')\n",
    "n_tokens_per_batch = batch_size * unroll_steps\n",
    "n_batches_per_epoch = int(n_train_tokens / n_tokens_per_batch)\n",
    "n_batches_total = options['n_epochs'] * n_batches_per_epoch\n",
    "\n",
    "init_state_tensors = model.init_lstm_state\n",
    "final_state_tensors = model.final_lstm_state\n",
    "\n",
    "char_inputs = 'char_cnn' in options\n",
    "if char_inputs:\n",
    "    max_chars = options['char_cnn']['max_characters_per_token']\n",
    "    feed_dict = {\n",
    "        model.tokens_characters: np.zeros(\n",
    "            [batch_size, unroll_steps, max_chars], dtype = np.int32\n",
    "        )\n",
    "    }\n",
    "\n",
    "else:\n",
    "    feed_dict = {model.token_ids: np.zeros([batch_size, unroll_steps])}\n",
    "\n",
    "if bidirectional:\n",
    "    if char_inputs:\n",
    "        feed_dict.update(\n",
    "            {\n",
    "                model.tokens_characters_reverse: np.zeros(\n",
    "                    [batch_size, unroll_steps, max_chars], dtype = np.int32\n",
    "                )\n",
    "            }\n",
    "        )\n",
    "    else:\n",
    "        feed_dict.update(\n",
    "            {\n",
    "                model.token_ids_reverse: np.zeros(\n",
    "                    [batch_size, unroll_steps], dtype = np.int32\n",
    "                )\n",
    "            }\n",
    "        )\n",
    "\n",
    "init_state_values = sess.run(init_state_tensors, feed_dict = feed_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "train minibatch loop: 100%|█████████████████████████████████████████| 14200/14200 [2:50:17<00:00,  1.40it/s, cost=3.54]\n"
     ]
    }
   ],
   "source": [
    "data_gen = bi.iter_batches(batch_size, unroll_steps)\n",
    "pbar = tqdm(range(n_batches_total), desc = 'train minibatch loop')\n",
    "for p in pbar:\n",
    "    batch = next(data_gen)\n",
    "    feed_dict = {t: v for t, v in zip(init_state_tensors, init_state_values)}\n",
    "    feed_dict.update(_get_feed_dict_from_X(batch, model, char_inputs, bidirectional))\n",
    "    score, loss, _, init_state_values = sess.run([model.output_scores,\n",
    "                                           model.total_loss, model.optimizer, final_state_tensors],\n",
    "            feed_dict = feed_dict)\n",
    "    pbar.set_postfix(cost = loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "word_embed = model.softmax_W.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from scipy.spatial.distance import cdist\n",
    "from sklearn.neighbors import NearestNeighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['心中', 0.41927003860473633],\n",
       " ['李莫愁', 0.40811800956726074],\n",
       " ['杨过', 0.39115822315216064],\n",
       " ['脸色', 0.37209492921829224],\n",
       " ['不知', 0.3664872646331787],\n",
       " ['二人', 0.36436891555786133],\n",
       " ['猛地', 0.3582932949066162],\n",
       " ['外', 0.3516373634338379],\n",
       " ['枣核', 0.34800589084625244]]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word = '金轮'\n",
    "nn = NearestNeighbors(10, metric = 'cosine').fit(word_embed)\n",
    "distances, idx = nn.kneighbors(word_embed[dictionary[word]].reshape((1, -1)))\n",
    "word_list = []\n",
    "for i in range(1, idx.shape[1]):\n",
    "    word_list.append([rev_dictionary[idx[0, i]], 1 - distances[0, i]])\n",
    "word_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
